import warnings
import matplotlib.cbook
warnings.filterwarnings("ignore",category=matplotlib.cbook.mplDeprecation)
import pickle
import multiprocessing as mp
import pandas as pd
pd.set_option('display.max_rows', None)
pd.set_option('display.max_columns', None)
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
plt.style.use('dark_background')
import seaborn as sns
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import StandardScaler
from sklearn.cluster import DBSCAN
from sklearn.neighbors import LocalOutlierFactor
from IPython.core.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>"))
from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = "all"
%cd D:\ABSOLUTELY_CRITICAL_BACKUP_FOLDER_LOCATION\
D:\ABSOLUTELY_CRITICAL_BACKUP_FOLDER_LOCATION
pf = pd.read_pickle("SSS.pkl")
# this pickle file was previously created from millions of rows
# of data and then filtered for removal of any NaN values,
# extensively cleaned and reproduced
# Data Time Domain
# Date Range: March 2018 - March 2021
# Granularity: 25s
# Nodes: Single Node Location
pf.head(10)
| sensor | parameter | value_hrf | |
|---|---|---|---|
| timestamp | |||
| 2018-03-06 22:14:56 | apds_9006_020 | intensity | 3.537000 |
| 2018-03-06 22:14:56 | apds_9006_020 | intensity | 3.537000 |
| 2018-03-06 22:14:56 | hih6130 | humidity | 61.830002 |
| 2018-03-06 22:14:56 | hih6130 | humidity | 61.830002 |
| 2018-03-06 22:14:56 | hih6130 | temperature | 9.620000 |
| 2018-03-06 22:14:56 | hih6130 | temperature | 9.620000 |
| 2018-03-06 22:14:56 | hmc5883l | magnetic_field_x | -148.182007 |
| 2018-03-06 22:14:56 | hmc5883l | magnetic_field_x | -150.000000 |
| 2018-03-06 22:14:56 | hmc5883l | magnetic_field_y | -1603.635986 |
| 2018-03-06 22:14:56 | hmc5883l | magnetic_field_y | -1604.545044 |
pf.tail()
| sensor | parameter | value_hrf | |
|---|---|---|---|
| timestamp | |||
| 2021-03-22 05:18:42 | pr103j2 | temperature | 14.000000 |
| 2021-03-22 05:18:42 | spv1840lr5h_b | intensity | 56.720001 |
| 2021-03-22 05:18:42 | tmp112 | temperature | 14.060000 |
| 2021-03-22 05:18:42 | tsl250rd | intensity | 0.000000 |
| 2021-03-22 05:18:42 | tsys01 | temperature | 14.290000 |
pf.dtypes
sensor object parameter object value_hrf float32 dtype: object
for p in pf.parameter.unique():print(p)
intensity humidity temperature magnetic_field_x magnetic_field_y magnetic_field_z pressure acceleration_x acceleration_y acceleration_z concentration
c=1
for s in pf.sensor.unique():
print(c,") ", s)
c = c+1
1 ) apds_9006_020 2 ) hih6130 3 ) hmc5883l 4 ) ml8511 5 ) mlx75305 6 ) tmp421 7 ) tsl250rd 8 ) tsl260rd 9 ) bmp180 10 ) hih4030 11 ) htu21d 12 ) mma8452q 13 ) pr103j2 14 ) spv1840lr5h_b 15 ) tmp112 16 ) tsys01 17 ) at0 18 ) at1 19 ) at2 20 ) at3 21 ) lps25h 22 ) sht25 23 ) co 24 ) h2s 25 ) no2 26 ) o3 27 ) oxidizing_gases 28 ) reducing_gases 29 ) so2
for p in pf.parameter.unique():print(p) # params
intensity humidity temperature magnetic_field_x magnetic_field_y magnetic_field_z pressure acceleration_x acceleration_y acceleration_z concentration
pf['sensor'] = pf.sensor.astype('category')
pf['parameter'] = pf.parameter.astype('category')
pf.dtypes
sensor category parameter category value_hrf float32 dtype: object
# len(pf)
# 82M rows... 82 406 931
# date range
# date range
pf.head(1).index[0]
pf.tail(1).index[0]
Timestamp('2018-03-06 22:14:56')
Timestamp('2021-03-22 05:18:42')
# range above is over 3 years of data, single node,
# multiple sub-sensors, taken every 25s !
# stop: you can re-use this code from here
# to spin off multiple sensor analysis if desired
# this is an excellent base dataframe to do more additional examination...
# but for now we will grab particular window
We will grab a particular window in 2019 of just over two months
We still have a fair amount of data (given sensors record every 25s)
- - - Create small window for speed, plotting numerous kpi values - - -
# from sklearn.cluster import DBSCAN
# clustering1 = DBSCAN(eps=0.09, min_samples=6).fit(np.array(ts_dataframe['Normalized Profit']).reshape(-1,1))
# labels = clustering1.labels_
# outlier_pos = np.where(labels == -1)[0]
# x = []; y = [];
# for pos in outlier_pos:
# x.append(np.array(ts_dataframe['Normalized Profit'])[pos])
# y.append(ts_dataframe['Normalized Profit'].index[pos])
# plt.plot(ts_dataframe['Normalized Profit'].loc[ts_dataframe['Normalized Profit'].index], 'k-')
# plt.plot(y,x,'r*', markersize=8)
# plt.legend(['Actual', 'Anomaly Detected'])
# plt.xlabel('Time Period')
# plt.xticks([0, 20, 40, 60, 80, 99],[ts_dataframe.index[0],ts_dataframe.index[20], ts_dataframe.index[40], ts_dataframe.index[60], ts_dataframe.index[80], ts_dataframe.index[99]] ,rotation=45)
# plt.ylabel('Normalized Profit')
# minidf = df.loc['2018-05-01 14:00:00':'2018-05-01 14:20:00'].copy()
minidf = pf.loc['2019-01-01':'2019-02-28'].copy()
# our base window will be a wide 2 month window to get
# the core dbscan algorithm working properly
# len(minidf) # 4,883,544 roughly 5M rows of data
minidf.head(49)
# every 25s, it adds up fast !
| sensor | parameter | value_hrf | |
|---|---|---|---|
| timestamp | |||
| 2019-01-01 00:00:03 | apds_9006_020 | intensity | 0.723000 |
| 2019-01-01 00:00:03 | hih6130 | humidity | 24.290001 |
| 2019-01-01 00:00:03 | hih6130 | temperature | 28.660000 |
| 2019-01-01 00:00:03 | hmc5883l | magnetic_field_x | -155.455002 |
| 2019-01-01 00:00:03 | hmc5883l | magnetic_field_y | -1548.182007 |
| 2019-01-01 00:00:03 | hmc5883l | magnetic_field_z | -521.429016 |
| 2019-01-01 00:00:03 | ml8511 | intensity | 46.351002 |
| 2019-01-01 00:00:03 | mlx75305 | intensity | 4.242000 |
| 2019-01-01 00:00:03 | tmp421 | temperature | 9.000000 |
| 2019-01-01 00:00:03 | tsl250rd | intensity | 23.672001 |
| 2019-01-01 00:00:03 | tsl260rd | intensity | 26.113001 |
| 2019-01-01 00:00:03 | bmp180 | pressure | 1020.489990 |
| 2019-01-01 00:00:03 | bmp180 | temperature | 7.300000 |
| 2019-01-01 00:00:03 | hih4030 | humidity | 83.050003 |
| 2019-01-01 00:00:03 | htu21d | humidity | 88.540001 |
| 2019-01-01 00:00:03 | htu21d | temperature | 4.830000 |
| 2019-01-01 00:00:03 | mma8452q | acceleration_x | -2.930000 |
| 2019-01-01 00:00:03 | mma8452q | acceleration_y | -998.046997 |
| 2019-01-01 00:00:03 | mma8452q | acceleration_z | -0.977000 |
| 2019-01-01 00:00:03 | pr103j2 | temperature | 4.350000 |
| 2019-01-01 00:00:03 | spv1840lr5h_b | intensity | 70.699997 |
| 2019-01-01 00:00:03 | tmp112 | temperature | 4.880000 |
| 2019-01-01 00:00:03 | tsl250rd | intensity | 0.000000 |
| 2019-01-01 00:00:03 | tsys01 | temperature | 4.800000 |
| 2019-01-01 00:00:28 | apds_9006_020 | intensity | 0.804000 |
| 2019-01-01 00:00:28 | hih6130 | humidity | 24.290001 |
| 2019-01-01 00:00:28 | hih6130 | temperature | 28.660000 |
| 2019-01-01 00:00:28 | hmc5883l | magnetic_field_x | -158.182007 |
| 2019-01-01 00:00:28 | hmc5883l | magnetic_field_y | -1550.909058 |
| 2019-01-01 00:00:28 | hmc5883l | magnetic_field_z | -504.082001 |
| 2019-01-01 00:00:28 | ml8511 | intensity | 46.370998 |
| 2019-01-01 00:00:28 | mlx75305 | intensity | 4.554000 |
| 2019-01-01 00:00:28 | tmp421 | temperature | 9.000000 |
| 2019-01-01 00:00:28 | tsl250rd | intensity | 23.674000 |
| 2019-01-01 00:00:28 | tsl260rd | intensity | 26.115000 |
| 2019-01-01 00:00:28 | bmp180 | pressure | 1020.510010 |
| 2019-01-01 00:00:28 | bmp180 | temperature | 7.300000 |
| 2019-01-01 00:00:28 | hih4030 | humidity | 83.349998 |
| 2019-01-01 00:00:28 | htu21d | humidity | 88.309998 |
| 2019-01-01 00:00:28 | htu21d | temperature | 4.760000 |
| 2019-01-01 00:00:28 | mma8452q | acceleration_x | 0.977000 |
| 2019-01-01 00:00:28 | mma8452q | acceleration_y | -998.046997 |
| 2019-01-01 00:00:28 | mma8452q | acceleration_z | 8.789000 |
| 2019-01-01 00:00:28 | pr103j2 | temperature | 4.250000 |
| 2019-01-01 00:00:28 | spv1840lr5h_b | intensity | 68.760002 |
| 2019-01-01 00:00:28 | tmp112 | temperature | 4.810000 |
| 2019-01-01 00:00:28 | tsl250rd | intensity | 0.000000 |
| 2019-01-01 00:00:28 | tsys01 | temperature | 4.720000 |
| 2019-01-01 00:00:53 | apds_9006_020 | intensity | 0.643000 |
# so at this point, i have a single node, and
# will grab a single sensor and build
# FILTER: smaller time window
# important
# grabbing the right accurate sensor for this was tricky
# i am focusing for now on the single dimension of temperature
tdf = minidf[minidf.parameter == 'temperature'].copy()
# you could go thru all the sensors and plot here,
# elements like temperature or whatever
tdf.head(8)
| sensor | parameter | value_hrf | |
|---|---|---|---|
| timestamp | |||
| 2019-01-01 00:00:03 | hih6130 | temperature | 28.66 |
| 2019-01-01 00:00:03 | tmp421 | temperature | 9.00 |
| 2019-01-01 00:00:03 | bmp180 | temperature | 7.30 |
| 2019-01-01 00:00:03 | htu21d | temperature | 4.83 |
| 2019-01-01 00:00:03 | pr103j2 | temperature | 4.35 |
| 2019-01-01 00:00:03 | tmp112 | temperature | 4.88 |
| 2019-01-01 00:00:03 | tsys01 | temperature | 4.80 |
| 2019-01-01 00:00:28 | hih6130 | temperature | 28.66 |
# FILTER: single sensor for now...
tsys = tdf[tdf.sensor == 'tsys01'].copy()
# the specific tsys01 sub-sensor for now
tsys.head(10)
| sensor | parameter | value_hrf | |
|---|---|---|---|
| timestamp | |||
| 2019-01-01 00:00:03 | tsys01 | temperature | 4.80 |
| 2019-01-01 00:00:28 | tsys01 | temperature | 4.72 |
| 2019-01-01 00:00:53 | tsys01 | temperature | 4.67 |
| 2019-01-01 00:01:18 | tsys01 | temperature | 4.68 |
| 2019-01-01 00:01:43 | tsys01 | temperature | 4.71 |
| 2019-01-01 00:02:08 | tsys01 | temperature | 4.68 |
| 2019-01-01 00:02:33 | tsys01 | temperature | 4.63 |
| 2019-01-01 00:02:58 | tsys01 | temperature | 4.72 |
| 2019-01-01 00:03:23 | tsys01 | temperature | 4.68 |
| 2019-01-01 00:03:48 | tsys01 | temperature | 4.64 |
# len(tsys) # 203 481
# so about 200,000 rows of data...
# del pf, tdf # save space
# do pf.info() and tdf.info()
tsys.head()
| sensor | parameter | value_hrf | |
|---|---|---|---|
| timestamp | |||
| 2019-01-01 00:00:03 | tsys01 | temperature | 4.80 |
| 2019-01-01 00:00:28 | tsys01 | temperature | 4.72 |
| 2019-01-01 00:00:53 | tsys01 | temperature | 4.67 |
| 2019-01-01 00:01:18 | tsys01 | temperature | 4.68 |
| 2019-01-01 00:01:43 | tsys01 | temperature | 4.71 |
tsys.tail()
| sensor | parameter | value_hrf | |
|---|---|---|---|
| timestamp | |||
| 2019-02-28 23:58:15 | tsys01 | temperature | -0.03 |
| 2019-02-28 23:58:40 | tsys01 | temperature | 0.15 |
| 2019-02-28 23:59:05 | tsys01 | temperature | 0.08 |
| 2019-02-28 23:59:30 | tsys01 | temperature | 0.05 |
| 2019-02-28 23:59:55 | tsys01 | temperature | -0.20 |
tsys.dtypes
sensor category parameter category value_hrf float32 dtype: object
tsys.value_hrf[:10]
timestamp 2019-01-01 00:00:03 4.80 2019-01-01 00:00:28 4.72 2019-01-01 00:00:53 4.67 2019-01-01 00:01:18 4.68 2019-01-01 00:01:43 4.71 2019-01-01 00:02:08 4.68 2019-01-01 00:02:33 4.63 2019-01-01 00:02:58 4.72 2019-01-01 00:03:23 4.68 2019-01-01 00:03:48 4.64 Name: value_hrf, dtype: float32
# CONVERT: C to F
# F = C×(9/5)+32
tsys['value_hrf'] = tsys['value_hrf'].apply(lambda x: (9/5)*x + 32)
tsys.value_hrf[:10]
timestamp 2019-01-01 00:00:03 40.640 2019-01-01 00:00:28 40.496 2019-01-01 00:00:53 40.406 2019-01-01 00:01:18 40.424 2019-01-01 00:01:43 40.478 2019-01-01 00:02:08 40.424 2019-01-01 00:02:33 40.334 2019-01-01 00:02:58 40.496 2019-01-01 00:03:23 40.424 2019-01-01 00:03:48 40.352 Name: value_hrf, dtype: float64
# PLOT: two month window with detailed title going forward (KEEP)
%cd D:\GITHUB_Repos\SensorAnalysis\ENTER\results
D:\GITHUB_Repos\SensorAnalysis\ENTER\results
plt.figure(figsize=(22, 9))
plt.plot(tsys.index, tsys.value_hrf, 'cyan', markersize=1, linewidth=.9, label="temp sensor")
# , 'k.', markersize=8)
# plt.legend(['Temp'])
# plt.xlabel('timestamp')
# plt.xticks([0, 20, 40, 60, 80, 99],[ts_dataframe.index[0],ts_dataframe.index[20], ts_dataframe.index[40], ts_dataframe.index[60], ts_dataframe.index[80], ts_dataframe.index[99]] ,rotation=45)
plt.ylabel('Temperature F°')
plt.box(False)
plt.title(r'Temperature Data - Chicago Sensor (time window: 2 months, sensor node identifier: 001e0610ee36, subsensor: tsys01)')
plt.grid(axis='y', color='grey', linestyle='-', linewidth=.5, alpha=.8)
plt.locator_params(axis='y', nbins=10)
plt.legend(loc = "upper right")
plt.tight_layout()
plt.show();
# plt.savefig('dbscan_single_subsensor_temp_data_plotted_two_months.png', dpi=600)
### DBSCAN user guide:
Help on class DBSCAN in module sklearn.cluster._dbscan:
class DBSCAN(sklearn.base.ClusterMixin, sklearn.base.BaseEstimator)
DBSCAN(eps=0.5, *, min_samples=5, metric='euclidean', metric_params=None, algorithm='auto', leaf_size=30, p=None, n_jobs=None)
Perform DBSCAN clustering from vector array or distance matrix.
DBSCAN - Density-Based Spatial Clustering of Applications with Noise.
Finds core samples of high density and expands clusters from them.
Good for data which contains clusters of similar density.
Read more in the :ref:`User Guide <dbscan>`.
Parameters
----------
eps : float, default=0.5
The maximum distance between two samples for one to be considered
as in the neighborhood of the other. This is not a maximum bound
on the distances of points within a cluster. This is the most
important DBSCAN parameter to choose appropriately for your data set
and distance function.
min_samples : int, default=5
The number of samples (or total weight) in a neighborhood for a point
to be considered as a core point. This includes the point itself.
metric : string, or callable, default='euclidean'
The metric to use when calculating distance between instances in a
feature array. If metric is a string or callable, it must be one of
the options allowed by :func:`sklearn.metrics.pairwise_distances` for
its metric parameter.
If metric is "precomputed", X is assumed to be a distance matrix and
must be square. X may be a :term:`Glossary <sparse graph>`, in which
case only "nonzero" elements may be considered neighbors for DBSCAN.
.. versionadded:: 0.17
metric *precomputed* to accept precomputed sparse matrix.
metric_params : dict, default=None
Additional keyword arguments for the metric function.
.. versionadded:: 0.19
algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, default='auto'
The algorithm to be used by the NearestNeighbors module
to compute pointwise distances and find nearest neighbors.
See NearestNeighbors module documentation for details.
leaf_size : int, default=30
Leaf size passed to BallTree or cKDTree. This can affect the speed
of the construction and query, as well as the memory required
to store the tree. The optimal value depends
on the nature of the problem.
p : float, default=None
The power of the Minkowski metric to be used to calculate distance
between points. If None, then ``p=2`` (equivalent to the Euclidean
distance).
n_jobs : int, default=None
The number of parallel jobs to run.
``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
``-1`` means using all processors. See :term:`Glossary <n_jobs>`
for more details.
Attributes
----------
core_sample_indices_ : ndarray of shape (n_core_samples,)
Indices of core samples.
components_ : ndarray of shape (n_core_samples, n_features)
Copy of each core sample found by training.
labels_ : ndarray of shape (n_samples)
Cluster labels for each point in the dataset given to fit().
Noisy samples are given the label -1.
Examples
--------
>>> from sklearn.cluster import DBSCAN
>>> import numpy as np
>>> X = np.array([[1, 2], [2, 2], [2, 3],
... [8, 7], [8, 8], [25, 80]])
>>> clustering = DBSCAN(eps=3, min_samples=2).fit(X)
>>> clustering.labels_
array([ 0, 0, 0, 1, 1, -1])
>>> clustering
DBSCAN(eps=3, min_samples=2)
See Also
--------
OPTICS : A similar clustering at multiple values of eps. Our implementation
is optimized for memory usage.
Notes
-----
For an example, see :ref:`examples/cluster/plot_dbscan.py
<sphx_glr_auto_examples_cluster_plot_dbscan.py>`.
This implementation bulk-computes all neighborhood queries, which increases
the memory complexity to O(n.d) where d is the average number of neighbors,
while original DBSCAN had memory complexity O(n). It may attract a higher
memory complexity when querying these nearest neighborhoods, depending
on the ``algorithm``.
One way to avoid the query complexity is to pre-compute sparse
neighborhoods in chunks using
:func:`NearestNeighbors.radius_neighbors_graph
<sklearn.neighbors.NearestNeighbors.radius_neighbors_graph>` with
``mode='distance'``, then using ``metric='precomputed'`` here.
Another way to reduce memory and computation time is to remove
(near-)duplicate points and use ``sample_weight`` instead.
:class:`cluster.OPTICS` provides a similar clustering with lower memory
usage.
References
----------
Ester, M., H. P. Kriegel, J. Sander, and X. Xu, "A Density-Based
Algorithm for Discovering Clusters in Large Spatial Databases with Noise".
In: Proceedings of the 2nd International Conference on Knowledge Discovery
and Data Mining, Portland, OR, AAAI Press, pp. 226-231. 1996
Schubert, E., Sander, J., Ester, M., Kriegel, H. P., & Xu, X. (2017).
DBSCAN revisited, revisited: why and how you should (still) use DBSCAN.
ACM Transactions on Database Systems (TODS), 42(3), 19.
Method resolution order:
DBSCAN
sklearn.base.ClusterMixin
sklearn.base.BaseEstimator
builtins.object
Methods defined here:
__init__(self, eps=0.5, *, min_samples=5, metric='euclidean', metric_params=None, algorithm='auto', leaf_size=30, p=None, n_jobs=None)
Initialize self. See help(type(self)) for accurate signature.
fit(self, X, y=None, sample_weight=None)
Perform DBSCAN clustering from features, or distance matrix.
Parameters
----------
X : {array-like, sparse matrix} of shape (n_samples, n_features), or (n_samples, n_samples)
Training instances to cluster, or distances between instances if
``metric='precomputed'``. If a sparse matrix is provided, it will
be converted into a sparse ``csr_matrix``.
sample_weight : array-like of shape (n_samples,), default=None
Weight of each sample, such that a sample with a weight of at least
``min_samples`` is by itself a core sample; a sample with a
negative weight may inhibit its eps-neighbor from being core.
Note that weights are absolute, and default to 1.
y : Ignored
Not used, present here for API consistency by convention.
Returns
-------
self
fit_predict(self, X, y=None, sample_weight=None)
Perform DBSCAN clustering from features or distance matrix,
and return cluster labels.
Parameters
----------
X : {array-like, sparse matrix} of shape (n_samples, n_features), or (n_samples, n_samples)
Training instances to cluster, or distances between instances if
``metric='precomputed'``. If a sparse matrix is provided, it will
be converted into a sparse ``csr_matrix``.
sample_weight : array-like of shape (n_samples,), default=None
Weight of each sample, such that a sample with a weight of at least
``min_samples`` is by itself a core sample; a sample with a
negative weight may inhibit its eps-neighbor from being core.
Note that weights are absolute, and default to 1.
y : Ignored
Not used, present here for API consistency by convention.
Returns
-------
labels : ndarray of shape (n_samples,)
Cluster labels. Noisy samples are given the label -1.
----------------------------------------------------------------------
Data descriptors inherited from sklearn.base.ClusterMixin:
__dict__
dictionary for instance variables (if defined)
__weakref__
list of weak references to the object (if defined)
----------------------------------------------------------------------
Methods inherited from sklearn.base.BaseEstimator:
__getstate__(self)
__repr__(self, N_CHAR_MAX=700)
Return repr(self).
__setstate__(self, state)
get_params(self, deep=True)
Get parameters for this estimator.
Parameters
----------
deep : bool, default=True
If True, will return the parameters for this estimator and
contained subobjects that are estimators.
Returns
-------
params : dict
Parameter names mapped to their values.
STOP:
# this time, i WONT use the standard scaler, just to see what we see...
# clustering1 = DBSCAN(eps=0.09, min_samples=6).fit(np.array(tsys['value_hrf']).
# reshape(-1,1)) # or you could take pandas series and go to df with to_frame() pandas function
from sklearn.cluster import DBSCAN
# eps=0.2, min_samples=8
# The argument 'eps' is the distance between two samples to be considered as a neighborhood and
# 'min_samples' is the number of samples in a neighborhood
####################################################################
db = DBSCAN(eps=0.1, min_samples=10, metric='euclidean', n_jobs=-1)
####################################################################
# X = StandardScaler().fit_transform(X)
# db = DBSCAN(eps=0.3, min_samples=10).fit(X)
db = db.fit(np.array(tsys['value_hrf']).reshape(-1, 1)) # i think smarter to do to_frame() instead...
db.get_params(deep=True)
{'algorithm': 'auto',
'eps': 0.1,
'leaf_size': 30,
'metric': 'euclidean',
'metric_params': None,
'min_samples': 10,
'n_jobs': -1,
'p': None}
label=db.labels_
from sklearn import metrics
#identifying the points which makes up our core points
sample_cores=np.zeros_like(label,dtype=bool)
sample_cores[db.core_sample_indices_]=True
# Calculating the number of clusters
n_clusters=len(set(label))- (1 if -1 in label else 0)
print('\nNo of clusters:',n_clusters, '\n\n')
# print('\nRemember, in theory 1 cluster actually means one normal cluster and everything else is _noise_')
tsys["cluster"] = db.labels_
print(tsys["cluster"].value_counts(), '\n')
#tsys["cluster"] = db.labels_
# tsys["cluster"].value_counts()
No of clusters: 5 0 198364 4 4990 1 66 -1 41 2 10 3 10 Name: cluster, dtype: int64
RESULTS:
========
db = DBSCAN(eps=0.03, min_samples=6, metric='euclidean', n_jobs=-1)
0 139811
3 4950
2 37
1 26
-1 19
4 10
db = DBSCAN(eps=0.04, min_samples=5, metric='euclidean', n_jobs=-1)
0 144840
-1 13
db = DBSCAN(eps=0.04, min_samples=7, metric='euclidean', n_jobs=-1)
0 139848
1 4989
-1 16
db = DBSCAN(eps=0.04, min_samples=7, metric='euclidean', n_jobs=-1)
0 139848
1 4989
-1 16
db = DBSCAN(eps=0.1, min_samples=10, metric='euclidean', n_jobs=-1)
0 198364
4 4990
1 66
-1 41
2 10
3 10
``
# much smaller time range now !!!!!!!
plt.figure(figsize=(25, 6))
plt.plot(tsys.index, tsys.value_hrf, 'grey', markersize=1, linewidth=.9)
# , 'k.', markersize=8)
plt.legend(['Temp'])
# plt.xlabel('timestamp')
# plt.xticks([0, 20, 40, 60, 80, 99],[ts_dataframe.index[0],ts_dataframe.index[20], ts_dataframe.index[40], ts_dataframe.index[60], ts_dataframe.index[80], ts_dataframe.index[99]] ,rotation=45)
plt.ylabel('Temperature F°')
plt.box(False)
plt.show();
tsys.head(10)
| sensor | parameter | value_hrf | cluster | |
|---|---|---|---|---|
| timestamp | ||||
| 2019-01-01 00:00:03 | tsys01 | temperature | 40.640 | 0 |
| 2019-01-01 00:00:28 | tsys01 | temperature | 40.496 | 0 |
| 2019-01-01 00:00:53 | tsys01 | temperature | 40.406 | 0 |
| 2019-01-01 00:01:18 | tsys01 | temperature | 40.424 | 0 |
| 2019-01-01 00:01:43 | tsys01 | temperature | 40.478 | 0 |
| 2019-01-01 00:02:08 | tsys01 | temperature | 40.424 | 0 |
| 2019-01-01 00:02:33 | tsys01 | temperature | 40.334 | 0 |
| 2019-01-01 00:02:58 | tsys01 | temperature | 40.496 | 0 |
| 2019-01-01 00:03:23 | tsys01 | temperature | 40.424 | 0 |
| 2019-01-01 00:03:48 | tsys01 | temperature | 40.352 | 0 |
# 0 139839
# 1 4987
# -1 18
# 2 9
# No of clusters: 3
# 0 139839
# 1 4987
# -1 18
# 2 9
# --- latest ---
# 0 198364
# 4 4990
# 1 66
# -1 41
# 2 10
# 3 10
# Name: cluster, dtype: int64
plt.style.use('dark_background')
plt.figure(figsize=(25, 7))
a = tsys.loc[tsys['cluster'] == -1].index
b= tsys[(tsys['cluster'] == -1)]['value_hrf']
plt.plot(tsys.index, tsys.value_hrf, 'white', markersize=1, linewidth=.9)
c = tsys.loc[tsys['cluster'] == 1].index
d= tsys[(tsys['cluster'] == 1)]['value_hrf']
e = tsys.loc[tsys['cluster'] == 2].index
f= tsys[(tsys['cluster'] == 2)]['value_hrf']
plt.scatter(e, f, color='orange', label = 'Anomaly Cluter 2', s=35, alpha=0.7)
# plt.scatter(a, b, color='red', label='Anomaly',s = 200, alpha=0.5)
plt.scatter(a, b, color='red', label = 'Anomaly Cluter -1', s=35, alpha=0.7)
plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 1', s=30, alpha=0.5)
# plt.xlabel('Arrival Date')
plt.ylabel('Temperature C°')
plt.legend()
plt.box(False)
plt.show();
# eps=0.2, min_samples=8
# The argument 'eps' is the distance between two samples to be considered as a neighborhood and
# 'min_samples' is the number of samples in a neighborhood
####################################################################
db = DBSCAN(eps=0.08, min_samples=10, metric='euclidean', n_jobs=-1)
####################################################################
db = db.fit(np.array(tsys['value_hrf']).reshape(-1, 1))
label=db.labels_
from sklearn import metrics
sample_cores=np.zeros_like(label,dtype=bool)
sample_cores[db.core_sample_indices_]=True
n_clusters=len(set(label))- (1 if -1 in label else 0)
print('\nNo of clusters:',n_clusters, '\n\n')
# print('\nRemember, in theory 1 cluster actually means one normal cluster and everything else is _noise_')
tsys["cluster"] = db.labels_
print(tsys["cluster"].value_counts(), '\n')
#tsys["cluster"] = db.labels_
# tsys["cluster"].value_counts()
No of clusters: 6 0 198355 5 4989 -1 74 1 28 4 20 2 10 3 5 Name: cluster, dtype: int64
plt.style.use('seaborn-poster')
plt.figure(figsize=(25, 8))
a = tsys.loc[tsys['cluster'] == -1].index
b= tsys[(tsys['cluster'] == -1)]['value_hrf']
plt.plot(tsys.index, tsys.value_hrf, 'black', markersize=1, linewidth=.9)
c = tsys.loc[tsys['cluster'] == 1].index
d= tsys[(tsys['cluster'] == 1)]['value_hrf']
e = tsys.loc[tsys['cluster'] == 2].index
f= tsys[(tsys['cluster'] == 2)]['value_hrf']
plt.scatter(e, f, color='orange', label = 'Anomaly Cluter 2', s=35, alpha=0.7)
# plt.scatter(a, b, color='red', label='Anomaly',s = 200, alpha=0.5)
plt.scatter(a, b, color='red', label = 'Anomaly Cluter -1', s=35, alpha=0.7)
plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 1', s=30, alpha=0.5)
# plt.xlabel('Arrival Date')
plt.ylabel('Temperature C°')
plt.legend()
plt.box(False)
# plt.show();
plt.savefig('db2.png', dpi=600);
START:
tsys.columns
Index(['sensor', 'parameter', 'value_hrf'], dtype='object')
# PYTHONIC:
thisFilter = tsys.filter('cluster')
tsys.drop(thisFilter, inplace=True, axis=1)
# tsys.drop('cluster', axis=1, inplace=True) # if you iterate, you need to remove !
tsys.columns
Index(['sensor', 'parameter', 'value_hrf'], dtype='object')
X = tsys['value_hrf'].copy()
X[:10]
timestamp 2019-01-01 00:00:03 40.640 2019-01-01 00:00:28 40.496 2019-01-01 00:00:53 40.406 2019-01-01 00:01:18 40.424 2019-01-01 00:01:43 40.478 2019-01-01 00:02:08 40.424 2019-01-01 00:02:33 40.334 2019-01-01 00:02:58 40.496 2019-01-01 00:03:23 40.424 2019-01-01 00:03:48 40.352 Name: value_hrf, dtype: float64
X.values[:10]
array([40.64000034, 40.49599962, 40.40600014, 40.42399969, 40.47800007,
40.42399969, 40.33400021, 40.49599962, 40.42399969, 40.35199976])
# Utilize standard scaler
X = StandardScaler().fit_transform(X.values.reshape(-1,1))
X[:10] # normalized ! ! ! !
array([[0.82499978],
[0.81444817],
[0.80785348],
[0.80917239],
[0.81312926],
[0.80917239],
[0.80257771],
[0.81444817],
[0.80917239],
[0.80389662]])
# len(X) # 203 481
plt.style.use('default')
plt.figure(figsize=(25, 7))
plt.plot(tsys.index, X, 'grey', markersize=1, linewidth=.9)
# plt.legend()
plt.box(False)
print("Standard Scaler View:")
plt.show();
# normalized
Standard Scaler View:
# clustering1 = DBSCAN(eps=0.09, min_samples=6).fit(np.array(tsys['value_hrf']).
# reshape(-1,1)) # or you could take pandas series and go to df with to_frame() pandas function
myeps = .01
myminsamples = 25
#************************************************************************************
db = DBSCAN(eps=myeps, min_samples=myminsamples, metric='euclidean', n_jobs=-1).fit(X)
#************************************************************************************
# db.get_params(deep=True)
# {'algorithm': 'auto',
# 'eps': 0.01,
# 'leaf_size': 30,
# 'metric': 'euclidean',
# 'metric_params': None,
# 'min_samples': 25,
# 'n_jobs': -1,
# 'p': None}
label=db.labels_
#identifying the points which makes up our core points
sample_cores=np.zeros_like(label,dtype=bool)
sample_cores[db.core_sample_indices_]=True
# Calculating the number of clusters
n_clusters=len(set(label))- (1 if -1 in label else 0)
print(myeps,myminsamples)
print('\nNo of clusters:',n_clusters, '\n')
# print('\nRemember, in theory 1 cluster actually means one normal cluster and everything else is _noise_')
tsys["cluster"] = db.labels_
print(tsys["cluster"].value_counts(), '\n')
#tsys["cluster"] = db.labels_
# tsys["cluster"].value_counts()
# tsys["cluster"] = db.labels_
# tsys["cluster"].value_counts()
0.01 25 No of clusters: 4 0 198355 3 4943 -1 141 1 30 2 12 Name: cluster, dtype: int64
plt.style.use('seaborn-ticks')
plt.figure(figsize=(25, 10))
plt.plot(tsys.index, tsys.value_hrf, 'grey', markersize=1, linewidth=.9)
a = tsys.loc[tsys['cluster'] == -1].index
b= tsys[(tsys['cluster'] == -1)]['value_hrf']
plt.scatter(a, b, color='red', label = 'Anomaly Cluter -1', s=50, alpha=0.6)
c = tsys.loc[tsys['cluster'] == 3].index
d= tsys[(tsys['cluster'] == 3)]['value_hrf']
plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 3', s=50, alpha=0.6)
e = tsys.loc[tsys['cluster'] == 1].index
f= tsys[(tsys['cluster'] == 1)]['value_hrf']
plt.scatter(e, f, color='green', label = 'Anomaly Cluter 1', s=50, alpha=0.6)
g = tsys.loc[tsys['cluster'] == 2].index
h= tsys[(tsys['cluster'] == 2)]['value_hrf']
plt.scatter(g, h, color='yellow', label = 'Anomaly Cluster 2', s=50, alpha=0.6)
# plt.xlabel('Arrival Date')
plt.ylabel('Temperature F°')
# plt.title(r'Temperature Data Anomalies - Chicago Sensor (alg: DBSCAN, time window: 2 months, sensor node identifier: 001e0610ee36, subsensor: tsys01)')
plt.title('Temp Data Anomalies 1Q-2019 [alg=DBSCAN, eps={0}, min_samples={1}]'.format(myeps, myminsamples))
# plt.legend()
plt.legend(loc='upper center', borderaxespad=0.)
plt.tight_layout()
plt.box(False)
# plt.show();
plt.savefig('dbrun_1.png', dpi=800);
#
# adding shade !!!!!!!!!!!!
plt.style.use('seaborn-ticks')
plt.figure(figsize=(25, 10))
plt.plot(tsys.index, tsys.value_hrf, 'grey', markersize=1, linewidth=.9)
a = tsys.loc[tsys['cluster'] == -1].index
b= tsys[(tsys['cluster'] == -1)]['value_hrf']
print('\nANOMALOUS REGION (-1):')
# for date in a.values:
# print(a)
for c in range(140):
print(a[c])
# print(a[0])
# print(a[1])
min_a = a.min()
max_a = a[-9]
print("")
print('Red Window Start:', min_a)
print('Red Window End: ', max_a)
print("")
plt.axvspan(min_a, max_a, alpha=.3, color='red')
# line1 = min_a
# line2 = max_a
# ax.axvspan(line1, line2, alpha=.7, color='red')
# plt.scatter(a, b, color='red', label = 'Anomaly Cluter -1', s=50, alpha=0.6)
c = tsys.loc[tsys['cluster'] == 3].index
d= tsys[(tsys['cluster'] == 3)]['value_hrf']
# plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 3', s=50, alpha=0.6)
print('ARCTIC POLAR VORTEX:')
min_c = c.min()
max_c = c.max()
print('Dominant Blue Window Start:', min_c)
print('Dominant Blue Window End: ', max_c)
line1 = min_c
line2 = max_c
plt.axvspan(line1, line2, alpha=.3, color='blue')
# plt.axvline(x=0.22058956)
# e = tsys.loc[tsys['cluster'] == 1].index
# f= tsys[(tsys['cluster'] == 1)]['value_hrf']
# plt.scatter(e, f, color='green', label = 'Anomaly Cluter 1', s=50, alpha=0.6)
# g = tsys.loc[tsys['cluster'] == 2].index
# h= tsys[(tsys['cluster'] == 2)]['value_hrf']
# plt.scatter(g, h, color='yellow', label = 'Anomaly Cluster 2', s=50, alpha=0.6)
plt.ylabel('Temperature F°')
plt.title('Temp Data Anomalies 1Q-2019 [alg=DBSCAN, eps={0}, min_samples={1}]'.format(myeps, myminsamples))
# plt.legend(loc='upper center', borderaxespad=0.)
plt.tight_layout()
plt.box(False)
plt.xlim(pd.Timestamp('2019-01-05 20:07:20'), pd.Timestamp('2019-01-31 16:26:33'))
# ax.set_xlim(pd.Timestamp('2015-02-15'), pd.Timestamp('2015-07-01'))
plt.show();
# plt.savefig('dbrun_1.png', dpi=800);
ANOMALOUS REGION (-1): 2019-01-05 20:07:20 2019-01-05 20:07:45 2019-01-05 20:08:11 2019-01-05 20:08:36 2019-01-05 20:09:01 2019-01-05 20:09:26 2019-01-05 20:09:51 2019-01-05 20:10:16 2019-01-05 20:10:41 2019-01-05 20:21:57 2019-01-05 20:22:22 2019-01-05 20:27:23 2019-01-05 20:27:48 2019-01-05 20:28:13 2019-01-05 20:28:38 2019-01-05 20:29:03 2019-01-05 20:29:28 2019-01-05 20:31:58 2019-01-05 20:32:23 2019-01-05 20:33:39 2019-01-05 20:34:04 2019-01-05 20:34:29 2019-01-05 20:34:54 2019-01-05 20:35:19 2019-01-05 20:35:44 2019-01-05 20:36:09 2019-01-05 20:36:34 2019-01-05 20:36:59 2019-01-05 20:37:24 2019-01-05 20:37:49 2019-01-05 20:38:14 2019-01-05 20:38:39 2019-01-05 20:39:04 2019-01-05 20:39:29 2019-01-05 20:39:54 2019-01-05 20:40:19 2019-01-05 20:40:44 2019-01-05 20:41:09 2019-01-05 20:41:34 2019-01-05 20:41:59 2019-01-05 20:42:25 2019-01-05 20:42:50 2019-01-05 20:43:15 2019-01-05 20:43:40 2019-01-05 20:44:05 2019-01-05 20:44:30 2019-01-05 20:44:55 2019-01-05 20:45:20 2019-01-05 20:45:45 2019-01-05 20:46:10 2019-01-05 20:46:35 2019-01-05 20:47:00 2019-01-05 20:47:25 2019-01-05 20:47:50 2019-01-05 20:48:15 2019-01-05 20:48:40 2019-01-05 20:49:05 2019-01-05 20:49:31 2019-01-05 20:49:56 2019-01-05 20:50:21 2019-01-05 20:50:46 2019-01-05 20:51:11 2019-01-05 20:51:36 2019-01-05 20:52:01 2019-01-05 20:52:26 2019-01-05 20:52:51 2019-01-05 20:53:16 2019-01-05 20:53:41 2019-01-05 20:54:06 2019-01-05 20:54:31 2019-01-05 20:54:56 2019-01-05 20:55:21 2019-01-05 20:55:46 2019-01-05 20:56:11 2019-01-05 20:56:36 2019-01-05 20:57:01 2019-01-05 20:57:26 2019-01-05 20:57:51 2019-01-05 20:58:16 2019-01-05 20:58:42 2019-01-05 20:59:07 2019-01-05 20:59:32 2019-01-05 20:59:57 2019-01-05 21:00:22 2019-01-05 21:00:47 2019-01-05 21:01:12 2019-01-05 21:01:37 2019-01-05 21:02:02 2019-01-05 21:02:27 2019-01-05 21:02:52 2019-01-05 21:03:17 2019-01-05 21:03:42 2019-01-05 21:04:07 2019-01-05 21:04:32 2019-01-05 21:04:57 2019-01-05 21:05:22 2019-01-05 21:05:47 2019-01-05 21:06:12 2019-01-05 21:06:38 2019-01-05 21:07:03 2019-01-05 21:07:28 2019-01-05 21:07:53 2019-01-05 21:08:18 2019-01-05 21:08:43 2019-01-05 21:09:08 2019-01-05 21:09:33 2019-01-05 21:09:58 2019-01-05 21:10:23 2019-01-05 21:10:48 2019-01-05 21:11:13 2019-01-05 21:11:38 2019-01-05 21:12:03 2019-01-05 21:12:28 2019-01-05 21:12:53 2019-01-05 21:13:18 2019-01-05 21:13:43 2019-01-05 21:14:08 2019-01-05 21:14:33 2019-01-05 21:14:58 2019-01-05 21:15:24 2019-01-05 21:15:49 2019-01-05 21:16:14 2019-01-05 21:16:39 2019-01-05 21:17:04 2019-01-05 21:24:35 2019-01-05 21:25:00 2019-01-05 21:25:25 2019-01-05 21:25:50 2019-01-05 21:35:26 2019-01-05 21:38:21 2019-01-05 21:38:46 2019-01-05 21:39:11 2019-01-05 21:39:36 2019-01-30 05:38:30 2019-01-30 05:38:55 2019-01-30 05:41:50 2019-01-30 05:44:20 2019-01-30 05:46:01 2019-01-30 05:47:41 2019-01-30 05:48:06 Red Window Start: 2019-01-05 20:07:20 Red Window End: 2019-01-05 21:39:36 ARCTIC POLAR VORTEX: Dominant Blue Window Start: 2019-01-30 05:49:46 Dominant Blue Window End: 2019-01-31 16:26:33
# plotting smaller easier to see regions
# adding shade !!!!!!!!!!!!
plt.style.use('seaborn-ticks')
plt.figure(figsize=(25, 10))
plt.plot(tsys.index, tsys.value_hrf, 'grey', markersize=1, linewidth=.9)
a = tsys.loc[tsys['cluster'] == -1].index
b= tsys[(tsys['cluster'] == -1)]['value_hrf']
print('\nANOMALOUS REGION (-1):')
# for date in a.values:
# print(a)
for c in range(140):
print(a[c])
# print(a[0])
# print(a[1])
min_a = a.min()
max_a = a[-9]
print("")
print('Red Window Start:', min_a)
print('Red Window End: ', max_a)
print("")
plt.axvspan(min_a, max_a, alpha=.2, color='red')
# line1 = min_a
# line2 = max_a
# ax.axvspan(line1, line2, alpha=.7, color='red')
# plt.scatter(a, b, color='red', label = 'Anomaly Cluter -1', s=50, alpha=0.6)
c = tsys.loc[tsys['cluster'] == 3].index
d= tsys[(tsys['cluster'] == 3)]['value_hrf']
# plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 3', s=50, alpha=0.6)
print('ARCTIC POLAR VORTEX:')
min_c = c.min()
max_c = c.max()
print('Dominant Blue Window Start:', min_c)
print('Dominant Blue Window End: ', max_c)
line1 = min_c
line2 = max_c
plt.axvspan(line1, line2, alpha=.3, color='blue')
# plt.axvline(x=0.22058956)
# e = tsys.loc[tsys['cluster'] == 1].index
# f= tsys[(tsys['cluster'] == 1)]['value_hrf']
# plt.scatter(e, f, color='green', label = 'Anomaly Cluter 1', s=50, alpha=0.6)
# g = tsys.loc[tsys['cluster'] == 2].index
# h= tsys[(tsys['cluster'] == 2)]['value_hrf']
# plt.scatter(g, h, color='yellow', label = 'Anomaly Cluster 2', s=50, alpha=0.6)
plt.ylabel('Temperature F°')
plt.title('Temp Data Anomalies 1Q-2019 [alg=DBSCAN, eps={0}, min_samples={1}]'.format(myeps, myminsamples))
# plt.legend(loc='upper center', borderaxespad=1)
# plt.tight_layout()
plt.box(False)
plt.xlim(pd.Timestamp('2019-01-05 19:45:00'), pd.Timestamp('2019-01-05 22:00:00'))
# ax.set_xlim(pd.Timestamp('2015-02-15'), pd.Timestamp('2015-07-01'))
plt.show();
# plt.savefig('dbrun_1.png', dpi=800);
ANOMALOUS REGION (-1): 2019-01-05 20:07:20 2019-01-05 20:07:45 2019-01-05 20:08:11 2019-01-05 20:08:36 2019-01-05 20:09:01 2019-01-05 20:09:26 2019-01-05 20:09:51 2019-01-05 20:10:16 2019-01-05 20:10:41 2019-01-05 20:21:57 2019-01-05 20:22:22 2019-01-05 20:27:23 2019-01-05 20:27:48 2019-01-05 20:28:13 2019-01-05 20:28:38 2019-01-05 20:29:03 2019-01-05 20:29:28 2019-01-05 20:31:58 2019-01-05 20:32:23 2019-01-05 20:33:39 2019-01-05 20:34:04 2019-01-05 20:34:29 2019-01-05 20:34:54 2019-01-05 20:35:19 2019-01-05 20:35:44 2019-01-05 20:36:09 2019-01-05 20:36:34 2019-01-05 20:36:59 2019-01-05 20:37:24 2019-01-05 20:37:49 2019-01-05 20:38:14 2019-01-05 20:38:39 2019-01-05 20:39:04 2019-01-05 20:39:29 2019-01-05 20:39:54 2019-01-05 20:40:19 2019-01-05 20:40:44 2019-01-05 20:41:09 2019-01-05 20:41:34 2019-01-05 20:41:59 2019-01-05 20:42:25 2019-01-05 20:42:50 2019-01-05 20:43:15 2019-01-05 20:43:40 2019-01-05 20:44:05 2019-01-05 20:44:30 2019-01-05 20:44:55 2019-01-05 20:45:20 2019-01-05 20:45:45 2019-01-05 20:46:10 2019-01-05 20:46:35 2019-01-05 20:47:00 2019-01-05 20:47:25 2019-01-05 20:47:50 2019-01-05 20:48:15 2019-01-05 20:48:40 2019-01-05 20:49:05 2019-01-05 20:49:31 2019-01-05 20:49:56 2019-01-05 20:50:21 2019-01-05 20:50:46 2019-01-05 20:51:11 2019-01-05 20:51:36 2019-01-05 20:52:01 2019-01-05 20:52:26 2019-01-05 20:52:51 2019-01-05 20:53:16 2019-01-05 20:53:41 2019-01-05 20:54:06 2019-01-05 20:54:31 2019-01-05 20:54:56 2019-01-05 20:55:21 2019-01-05 20:55:46 2019-01-05 20:56:11 2019-01-05 20:56:36 2019-01-05 20:57:01 2019-01-05 20:57:26 2019-01-05 20:57:51 2019-01-05 20:58:16 2019-01-05 20:58:42 2019-01-05 20:59:07 2019-01-05 20:59:32 2019-01-05 20:59:57 2019-01-05 21:00:22 2019-01-05 21:00:47 2019-01-05 21:01:12 2019-01-05 21:01:37 2019-01-05 21:02:02 2019-01-05 21:02:27 2019-01-05 21:02:52 2019-01-05 21:03:17 2019-01-05 21:03:42 2019-01-05 21:04:07 2019-01-05 21:04:32 2019-01-05 21:04:57 2019-01-05 21:05:22 2019-01-05 21:05:47 2019-01-05 21:06:12 2019-01-05 21:06:38 2019-01-05 21:07:03 2019-01-05 21:07:28 2019-01-05 21:07:53 2019-01-05 21:08:18 2019-01-05 21:08:43 2019-01-05 21:09:08 2019-01-05 21:09:33 2019-01-05 21:09:58 2019-01-05 21:10:23 2019-01-05 21:10:48 2019-01-05 21:11:13 2019-01-05 21:11:38 2019-01-05 21:12:03 2019-01-05 21:12:28 2019-01-05 21:12:53 2019-01-05 21:13:18 2019-01-05 21:13:43 2019-01-05 21:14:08 2019-01-05 21:14:33 2019-01-05 21:14:58 2019-01-05 21:15:24 2019-01-05 21:15:49 2019-01-05 21:16:14 2019-01-05 21:16:39 2019-01-05 21:17:04 2019-01-05 21:24:35 2019-01-05 21:25:00 2019-01-05 21:25:25 2019-01-05 21:25:50 2019-01-05 21:35:26 2019-01-05 21:38:21 2019-01-05 21:38:46 2019-01-05 21:39:11 2019-01-05 21:39:36 2019-01-30 05:38:30 2019-01-30 05:38:55 2019-01-30 05:41:50 2019-01-30 05:44:20 2019-01-30 05:46:01 2019-01-30 05:47:41 2019-01-30 05:48:06 Red Window Start: 2019-01-05 20:07:20 Red Window End: 2019-01-05 21:39:36 ARCTIC POLAR VORTEX: Dominant Blue Window Start: 2019-01-30 05:49:46 Dominant Blue Window End: 2019-01-31 16:26:33
# adding shade !!!!!!!!!!!!
plt.style.use('seaborn-ticks')
plt.figure(figsize=(25, 10))
plt.plot(tsys.index, tsys.value_hrf, 'grey', markersize=1, linewidth=.9)
a = tsys.loc[tsys['cluster'] == -1].index
b= tsys[(tsys['cluster'] == -1)]['value_hrf']
print('\nANOMALOUS REGION (-1):')
# for date in a.values:
# print(a)
for c in range(140):
print(a[c])
# print(a[0])
# print(a[1])
min_a = a.min()
max_a = a[-9]
print("")
print('Red Window Start:', min_a)
print('Red Window End: ', max_a)
print("")
plt.axvspan(min_a, max_a, alpha=.3, color='red')
# line1 = min_a
# line2 = max_a
# ax.axvspan(line1, line2, alpha=.7, color='red')
# plt.scatter(a, b, color='red', label = 'Anomaly Cluter -1', s=50, alpha=0.6)
c = tsys.loc[tsys['cluster'] == 3].index
d= tsys[(tsys['cluster'] == 3)]['value_hrf']
# plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 3', s=50, alpha=0.6)
print('ARCTIC POLAR VORTEX:')
min_c = c.min()
max_c = c.max()
print('Dominant Blue Window Start:', min_c)
print('Dominant Blue Window End: ', max_c)
line1 = min_c
line2 = max_c
plt.axvspan(line1, line2, alpha=.3, color='blue')
# plt.axvline(x=0.22058956)
# e = tsys.loc[tsys['cluster'] == 1].index
# f= tsys[(tsys['cluster'] == 1)]['value_hrf']
# plt.scatter(e, f, color='green', label = 'Anomaly Cluter 1', s=50, alpha=0.6)
# g = tsys.loc[tsys['cluster'] == 2].index
# h= tsys[(tsys['cluster'] == 2)]['value_hrf']
# plt.scatter(g, h, color='yellow', label = 'Anomaly Cluster 2', s=50, alpha=0.6)
plt.ylabel('Temperature F°')
plt.title('Temp Data Anomalies 1Q-2019 [alg=DBSCAN, eps={0}, min_samples={1}]'.format(myeps, myminsamples))
# plt.legend(loc='upper center', borderaxespad=0.)
plt.tight_layout()
plt.box(False)
plt.xlim(pd.Timestamp('2019-01-05 20:07:20'), pd.Timestamp('2019-01-31 16:26:33'))
# ax.set_xlim(pd.Timestamp('2015-02-15'), pd.Timestamp('2015-07-01'))
plt.show();
# plt.savefig('dbrun_1.png', dpi=800);
ANOMALOUS REGION (-1): 2019-01-05 20:07:20 2019-01-05 20:07:45 2019-01-05 20:08:11 2019-01-05 20:08:36 2019-01-05 20:09:01 2019-01-05 20:09:26 2019-01-05 20:09:51 2019-01-05 20:10:16 2019-01-05 20:10:41 2019-01-05 20:21:57 2019-01-05 20:22:22 2019-01-05 20:27:23 2019-01-05 20:27:48 2019-01-05 20:28:13 2019-01-05 20:28:38 2019-01-05 20:29:03 2019-01-05 20:29:28 2019-01-05 20:31:58 2019-01-05 20:32:23 2019-01-05 20:33:39 2019-01-05 20:34:04 2019-01-05 20:34:29 2019-01-05 20:34:54 2019-01-05 20:35:19 2019-01-05 20:35:44 2019-01-05 20:36:09 2019-01-05 20:36:34 2019-01-05 20:36:59 2019-01-05 20:37:24 2019-01-05 20:37:49 2019-01-05 20:38:14 2019-01-05 20:38:39 2019-01-05 20:39:04 2019-01-05 20:39:29 2019-01-05 20:39:54 2019-01-05 20:40:19 2019-01-05 20:40:44 2019-01-05 20:41:09 2019-01-05 20:41:34 2019-01-05 20:41:59 2019-01-05 20:42:25 2019-01-05 20:42:50 2019-01-05 20:43:15 2019-01-05 20:43:40 2019-01-05 20:44:05 2019-01-05 20:44:30 2019-01-05 20:44:55 2019-01-05 20:45:20 2019-01-05 20:45:45 2019-01-05 20:46:10 2019-01-05 20:46:35 2019-01-05 20:47:00 2019-01-05 20:47:25 2019-01-05 20:47:50 2019-01-05 20:48:15 2019-01-05 20:48:40 2019-01-05 20:49:05 2019-01-05 20:49:31 2019-01-05 20:49:56 2019-01-05 20:50:21 2019-01-05 20:50:46 2019-01-05 20:51:11 2019-01-05 20:51:36 2019-01-05 20:52:01 2019-01-05 20:52:26 2019-01-05 20:52:51 2019-01-05 20:53:16 2019-01-05 20:53:41 2019-01-05 20:54:06 2019-01-05 20:54:31 2019-01-05 20:54:56 2019-01-05 20:55:21 2019-01-05 20:55:46 2019-01-05 20:56:11 2019-01-05 20:56:36 2019-01-05 20:57:01 2019-01-05 20:57:26 2019-01-05 20:57:51 2019-01-05 20:58:16 2019-01-05 20:58:42 2019-01-05 20:59:07 2019-01-05 20:59:32 2019-01-05 20:59:57 2019-01-05 21:00:22 2019-01-05 21:00:47 2019-01-05 21:01:12 2019-01-05 21:01:37 2019-01-05 21:02:02 2019-01-05 21:02:27 2019-01-05 21:02:52 2019-01-05 21:03:17 2019-01-05 21:03:42 2019-01-05 21:04:07 2019-01-05 21:04:32 2019-01-05 21:04:57 2019-01-05 21:05:22 2019-01-05 21:05:47 2019-01-05 21:06:12 2019-01-05 21:06:38 2019-01-05 21:07:03 2019-01-05 21:07:28 2019-01-05 21:07:53 2019-01-05 21:08:18 2019-01-05 21:08:43 2019-01-05 21:09:08 2019-01-05 21:09:33 2019-01-05 21:09:58 2019-01-05 21:10:23 2019-01-05 21:10:48 2019-01-05 21:11:13 2019-01-05 21:11:38 2019-01-05 21:12:03 2019-01-05 21:12:28 2019-01-05 21:12:53 2019-01-05 21:13:18 2019-01-05 21:13:43 2019-01-05 21:14:08 2019-01-05 21:14:33 2019-01-05 21:14:58 2019-01-05 21:15:24 2019-01-05 21:15:49 2019-01-05 21:16:14 2019-01-05 21:16:39 2019-01-05 21:17:04 2019-01-05 21:24:35 2019-01-05 21:25:00 2019-01-05 21:25:25 2019-01-05 21:25:50 2019-01-05 21:35:26 2019-01-05 21:38:21 2019-01-05 21:38:46 2019-01-05 21:39:11 2019-01-05 21:39:36 2019-01-30 05:38:30 2019-01-30 05:38:55 2019-01-30 05:41:50 2019-01-30 05:44:20 2019-01-30 05:46:01 2019-01-30 05:47:41 2019-01-30 05:48:06 Red Window Start: 2019-01-05 20:07:20 Red Window End: 2019-01-05 21:39:36 ARCTIC POLAR VORTEX: Dominant Blue Window Start: 2019-01-30 05:49:46 Dominant Blue Window End: 2019-01-31 16:26:33
# CRITICAL, DO NOT LOSE THIS CODE !
# use dark background !!!!!!!!!!!!!!!!!!!!!!
plt.style.use('seaborn-ticks')
fig, ax = plt.subplots(figsize=(30,8))
# plt.plot(tsys.index, tsys.value_hrf, 'grey', markersize=1, linewidth=.9)
# plt.style.use('white_background')
plt.plot(tsys.index, tsys.value_hrf, color = 'grey', alpha = .7, markersize=.8, linewidth=.9, label="temp sensor")
# --- original code
# a = tsys.loc[tsys['cluster'] == -1].index
# b= tsys[(tsys['cluster'] == -1)]['value_hrf']
# plt.scatter(a, b, color='red', label = 'Anomaly Cluter -1', s=20, alpha=0.9)
# c = tsys.loc[tsys['cluster'] == 1].index
# d= tsys[(tsys['cluster'] == 1)]['value_hrf']
# plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 1', s=20, alpha=0.9)
# e = tsys.loc[tsys['cluster'] == 2].index
# f= tsys[(tsys['cluster'] == 2)]['value_hrf']
# plt.scatter(e, f, color='orange', label = 'Anomaly Cluter 2', s=20, alpha=0.9)
# g = tsys.loc[tsys['cluster'] == 3].index
# h= tsys[(tsys['cluster'] == 3)]['value_hrf']
# plt.scatter(g, h, color='yellow', label = 'Anomaly Cluster 3', s=20, alpha=0.9)
a = tsys.loc[tsys['cluster'] == -1].index
b= tsys[(tsys['cluster'] == -1)]['value_hrf']
plt.scatter(a, b, color='red', label = 'Anomaly Cluter -1', s=50, alpha=0.6)
c = tsys.loc[tsys['cluster'] == 3].index
d= tsys[(tsys['cluster'] == 3)]['value_hrf']
plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 3', s=50, alpha=0.6)
e = tsys.loc[tsys['cluster'] == 1].index
f= tsys[(tsys['cluster'] == 1)]['value_hrf']
plt.scatter(e, f, color='green', label = 'Anomaly Cluter 1', s=50, alpha=0.6)
g = tsys.loc[tsys['cluster'] == 2].index
h= tsys[(tsys['cluster'] == 2)]['value_hrf']
plt.scatter(g, h, color='yellow', label = 'Anomaly Cluster 2', s=50, alpha=0.6)
# ax.annotate('local maximum', xy=('2019-01-01', 10), xytext=('2019-01-29', 5),
# color="white", arrowprops=dict(facecolor='green', shrink=0.05))
# ax.annotate("Independence Day", xy=('2019-01-29', -20), xycoords='data',
# bbox=dict(boxstyle="round", fc="none", ec="gray"),
# xytext=(10, -40), textcoords='offset points', ha='center',
# arrowprops=dict(arrowstyle="->"))
ax.annotate("Unusually High Winter Temps", xy=(pd.Timestamp('2019-01-06'), 60), xycoords='data',
bbox=dict(boxstyle="round", fc="none", ec="red"),
xytext=(130,20), textcoords='offset points', ha='center',
arrowprops=dict(arrowstyle="->"))
ax.annotate("Polar Vortex Storm Hits Chicago !", xy=(pd.Timestamp('2019-01-30'), -12), xycoords='data',
bbox=dict(boxstyle="round", fc="none", ec="yellow"),
xytext=(-140, 20), textcoords='offset points', ha='center',
arrowprops=dict(arrowstyle="->"))
# https://jakevdp.github.io/PythonDataScienceHandbook/04.09-text-and-annotation.html
plt.title(r'Temperature Data Anomalies - Chicago Sensor (alg: DBSCAN, time window: 2 months, sensor node identifier: 001e0610ee36, subsensor: tsys01)')
plt.grid(axis='y', color='grey', linestyle='--', linewidth=.5, alpha=.3)
plt.locator_params(axis='y', nbins=10)
plt.locator_params(axis='x', nbins=20)
plt.legend(loc='upper center', borderaxespad=0.)
plt.ylabel('Temperature F°')
plt.box(False)
plt.tight_layout()
plt.show();
# plt.savefig('single_subsensor_temp_data_two_month_dbscan_clusters.png', dpi=600)
C:\Users\tbresee\AppData\Local\Temp/ipykernel_22268/2381870372.py:49: UserWarning: 'set_params()' not defined for locator of type <class 'matplotlib.dates.AutoDateLocator'> plt.locator_params(axis='x', nbins=20)
# clusters seem to form near the yellow... but hard to see...
# clustering1 = DBSCAN(eps=0.09, min_samples=6).fit(np.array(tsys['value_hrf']).
# reshape(-1,1)) # or you could take pandas series and go to df with to_frame() pandas function
myeps = .01
myminsamples = 25
#************************************************************************************
db = DBSCAN(eps=myeps, min_samples=myminsamples, metric='euclidean', n_jobs=-1).fit(X)
#************************************************************************************
# db.get_params(deep=True)
# {'algorithm': 'auto',
# 'eps': 0.01,
# 'leaf_size': 30,
# 'metric': 'euclidean',
# 'metric_params': None,
# 'min_samples': 25,
# 'n_jobs': -1,
# 'p': None}
label=db.labels_
#identifying the points which makes up our core points
sample_cores=np.zeros_like(label,dtype=bool)
sample_cores[db.core_sample_indices_]=True
# Calculating the number of clusters
n_clusters=len(set(label))- (1 if -1 in label else 0)
print(myeps,myminsamples)
print('\nNo of clusters:',n_clusters, '\n')
# print('\nRemember, in theory 1 cluster actually means one normal cluster and everything else is _noise_')
tsys["cluster"] = db.labels_
print(tsys["cluster"].value_counts(), '\n')
#tsys["cluster"] = db.labels_
# tsys["cluster"].value_counts()
# tsys["cluster"] = db.labels_
# tsys["cluster"].value_counts()
0.01 25 No of clusters: 4 0 198355 3 4943 -1 141 1 30 2 12 Name: cluster, dtype: int64
plt.style.use('seaborn-ticks')
plt.figure(figsize=(25, 8))
plt.plot(tsys.index, tsys.value_hrf, 'grey', markersize=1, linewidth=.9)
a = tsys.loc[tsys['cluster'] == -1].index
b= tsys[(tsys['cluster'] == -1)]['value_hrf']
plt.scatter(a, b, color='red', label = 'Anomaly Cluter -1', s=50, alpha=0.6)
c = tsys.loc[tsys['cluster'] == 3].index
d= tsys[(tsys['cluster'] == 3)]['value_hrf']
plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 3', s=50, alpha=0.6)
e = tsys.loc[tsys['cluster'] == 1].index
f= tsys[(tsys['cluster'] == 1)]['value_hrf']
plt.scatter(e, f, color='orange', label = 'Anomaly Cluter 1', s=50, alpha=0.6)
g = tsys.loc[tsys['cluster'] == 2].index
h= tsys[(tsys['cluster'] == 2)]['value_hrf']
plt.scatter(g, h, color='yellow', label = 'Anomaly Cluster 2', s=50, alpha=0.6)
# plt.xlabel('Arrival Date')
plt.ylabel('Temperature F°')
# plt.title(r'Temperature Data Anomalies - Chicago Sensor (alg: DBSCAN, time window: 2 months, sensor node identifier: 001e0610ee36, subsensor: tsys01)')
plt.title('Temp Data Anomalies, eps = {}'.format(myeps))
plt.legend()
plt.tight_layout()
plt.box(False)
plt.show();
# use dark background !!!!!!!!!!!!!!!!!!!!!!
fig, ax = plt.subplots(figsize=(30,8))
# plt.style.use('white_background')
# plt.figure(figsize=(30, 8))
plt.plot(tsys.index, tsys.value_hrf, color = 'grey', alpha = .7, markersize=.8, linewidth=.9, label="temp sensor")
a = tsys.loc[tsys['cluster'] == -1].index
b= tsys[(tsys['cluster'] == -1)]['value_hrf']
plt.scatter(a, b, color='red', label = 'Anomaly Cluter -1', s=20, alpha=0.9)
c = tsys.loc[tsys['cluster'] == 1].index
d= tsys[(tsys['cluster'] == 1)]['value_hrf']
plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 1', s=20, alpha=0.9)
e = tsys.loc[tsys['cluster'] == 2].index
f= tsys[(tsys['cluster'] == 2)]['value_hrf']
plt.scatter(e, f, color='orange', label = 'Anomaly Cluter 2', s=20, alpha=0.9)
g = tsys.loc[tsys['cluster'] == 3].index
h= tsys[(tsys['cluster'] == 3)]['value_hrf']
plt.scatter(g, h, color='yellow', label = 'Anomaly Cluster 3', s=20, alpha=0.9)
# ax.annotate('local maximum', xy=('2019-01-01', 10), xytext=('2019-01-29', 5),
# color="white", arrowprops=dict(facecolor='green', shrink=0.05))
# ax.annotate("Independence Day", xy=('2019-01-29', -20), xycoords='data',
# bbox=dict(boxstyle="round", fc="none", ec="gray"),
# xytext=(10, -40), textcoords='offset points', ha='center',
# arrowprops=dict(arrowstyle="->"))
ax.annotate("Unusually High Winter Temps", xy=(pd.Timestamp('2019-01-06'), 60), xycoords='data',
bbox=dict(boxstyle="round", fc="none", ec="red"),
xytext=(130,20), textcoords='offset points', ha='center',
arrowprops=dict(arrowstyle="->"))
ax.annotate("Polar Vortex Storm Hits Chicago !", xy=(pd.Timestamp('2019-01-30'), -12), xycoords='data',
bbox=dict(boxstyle="round", fc="none", ec="yellow"),
xytext=(-140, 20), textcoords='offset points', ha='center',
arrowprops=dict(arrowstyle="->"))
# https://jakevdp.github.io/PythonDataScienceHandbook/04.09-text-and-annotation.html
plt.title(r'Temperature Data Anomalies - Chicago Sensor (alg: DBSCAN, time window: 2 months, sensor node identifier: 001e0610ee36, subsensor: tsys01)')
plt.grid(axis='y', color='grey', linestyle='--', linewidth=.5, alpha=.3)
plt.locator_params(axis='y', nbins=10)
plt.locator_params(axis='x', nbins=20)
plt.legend(loc = "upper left")
plt.ylabel('Temperature F°')
plt.box(False)
plt.tight_layout()
# plt.show();
plt.savefig('single_subsensor_temp_data_two_month_dbscan_clusters.png', dpi=600)
C:\Users\tbresee\AppData\Local\Temp/ipykernel_20040/594806885.py:49: UserWarning: 'set_params()' not defined for locator of type <class 'matplotlib.dates.AutoDateLocator'> plt.locator_params(axis='x', nbins=20)
Five records were set for Chicago for the month of January 2019:
plt.figure(figsize=(20, 6))
plt.scatter(g, h, color='grey', label = 'Anomaly Cluster 3', s=5, alpha=0.5)
plt.show();
# db = DBSCAN(eps=0.01, min_samples=25, metric='euclidean', n_jobs=-1).fit(X)
plt.figure(figsize=(35, 50))
plt.plot(tsys.index, tsys.value_hrf, 'grey', markersize=1, linewidth=.9)
a = tsys.loc[tsys['cluster'] == -1].index
b= tsys[(tsys['cluster'] == -1)]['value_hrf']
c = tsys.loc[tsys['cluster'] == 1].index
d= tsys[(tsys['cluster'] == 1)]['value_hrf']
e = tsys.loc[tsys['cluster'] == 2].index
f= tsys[(tsys['cluster'] == 2)]['value_hrf']
g = tsys.loc[tsys['cluster'] == 3].index
h= tsys[(tsys['cluster'] == 3)]['value_hrf']
plt.scatter(e, f, color='orange', label = 'Anomaly Cluter 2', s=50, alpha=0.6)
plt.scatter(a, b, color='red', label = 'Anomaly Cluter -1', s=50, alpha=0.6)
plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 1', s=50, alpha=0.6)
plt.scatter(g, h, color='black', label = 'Anomaly Cluster 3', s=50, alpha=0.6)
# plt.xlabel('Arrival Date')
plt.ylabel('Temperature C°')
plt.legend()
plt.box(False)
plt.show();
# The lowest temperature at Chicago was -23 degrees observed at Chicago O'Hare International Airport on January 30th.
# The lowest temperature at Rockford was -31 degrees observed at Chicago Rockford International Airport on January 31st.
# The lowest temperature at NWS Chicago in Romeoville was -24 degrees. This ties the record set on January 16, 2009.
# The lowest temperature at Chicago Midway Airport 3SW observed was -22 degrees on January 31, 2019.
plt.figure(figsize=(20, 6))
plt.scatter(g, h, color='blue', label = 'Anomaly Cluster 3', s=5, alpha=0.5)
plt.show();
# region cluster of anomalous temperature ! ! !
plt.figure(figsize=(20, 20))
plt.scatter(e, f, color='orange', label = 'Anomaly Cluter 2', s=50, alpha=0.6)
plt.scatter(a, b, color='red', label = 'Anomaly Cluter -1', s=50, alpha=0.6)
plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 1', s=50, alpha=0.6)
plt.show();
plt.figure(figsize=(20, 20))
plt.scatter(e, f, color='orange', label = 'Anomaly Cluter 2', s=50, alpha=0.6)
plt.scatter(a, b, color='red', label = 'Anomaly Cluter -1', s=50, alpha=0.6)
plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 1', s=50, alpha=0.6)
plt.show();
plt.figure(figsize=(25, 5))
plt.plot(tsys.index, tsys.value_hrf, 'grey', markersize=1, linewidth=.9)
a = tsys.loc[tsys['cluster'] == -1].index
b= tsys[(tsys['cluster'] == -1)]['value_hrf']
c = tsys.loc[tsys['cluster'] == 1].index
d= tsys[(tsys['cluster'] == 1)]['value_hrf']
# e = tsys.loc[tsys['cluster'] == 2].index
# f= tsys[(tsys['cluster'] == 2)]['value_hrf']
# plt.scatter(e, f, color='orange', label = 'Anomaly Cluter 2', s=35, alpha=0.7)
plt.scatter(a, b, color='red', label = 'Anomaly Cluter -1', s=35, alpha=0.7)
plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 1', s=30, alpha=0.5)
# plt.xlabel('Arrival Date')
plt.ylabel('Temperature C°')
plt.legend()
plt.box(False)
plt.show();
RESULTS:
db = DBSCAN(eps=0.01, min_samples=30, metric='euclidean', n_jobs=-1).fit(X)
0 139272
3 4284
4 664
2 403
-1 126
1 104
BELOW SHOWN IN PLOT:
db = DBSCAN(eps=0.01, min_samples=20, metric='euclidean', n_jobs=-1).fit(X)
0 139842
1 4988
-1 23
NOT BAD !
# ORIGINAL CODE BEFORE I USED THE NEW ABOVE
# label=db.labels_
# from sklearn import metrics
# #identifying the points which makes up our core points
# sample_cores=np.zeros_like(label,dtype=bool)
# sample_cores[db.core_sample_indices_]=True
# # Calculating the number of clusters
# n_clusters=len(set(label))- (1 if -1 in label else 0)
# print('No of clusters:',n_clusters)
# core_samples_mask = np.zeros_like(db.labels_, dtype=bool)
# core_samples_mask[db.core_sample_indices_] = True
# labels = db.labels_
# # Number of clusters in labels, ignoring noise if present.
# n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0)
# n_noise_ = list(labels).count(-1)
# print('Estimated number of clusters: %d' % n_clusters_)
# # print('Estimated number of noise points: %d' % n_noise_)
# # print("Homogeneity: %0.3f" % metrics.homogeneity_score(labels_true, labels))
# # print("Completeness: %0.3f" % metrics.completeness_score(labels_true, labels))
# # print("V-measure: %0.3f" % metrics.v_measure_score(labels_true, labels))
# print("Adjusted Rand Index: %0.3f"
# % metrics.adjusted_rand_score(labels_true, labels))
# print("Adjusted Mutual Information: %0.3f"
# % metrics.adjusted_mutual_info_score(labels_true, labels))
# print("Silhouette Coefficient: %0.3f"
# % metrics.silhouette_score(X, labels))
# EVERYTHING BELOW THIS LINE STILL WORKING ON:
from sklearn.neighbors import NearestNeighbors
nearest_neighbors = NearestNeighbors(n_neighbors=11)
neighbors = nearest_neighbors.fit(X)
distances, indices = neighbors.kneighbors(X)
distances = np.sort(distances[:,10], axis=0)
fig = plt.figure(figsize=(5, 5))
plt.plot(distances)
plt.xlabel("Points")
plt.ylabel("Distance")
plt.savefig("Distance_curve.png", dpi=300)
from sklearn.cluster import DBSCAN
clustering1 = DBSCAN(eps=0.09, min_samples=6).fit(np.array(ts_dataframe['Normalized Profit']).reshape(-1,1))
labels = clustering1.labels_
outlier_pos = np.where(labels == -1)[0]
x = []; y = [];
for pos in outlier_pos:
x.append(np.array(ts_dataframe['Normalized Profit'])[pos])
y.append(ts_dataframe['Normalized Profit'].index[pos])
plt.plot(ts_dataframe['Normalized Profit'].loc[ts_dataframe['Normalized Profit'].index], 'k-')
plt.plot(y,x,'r*', markersize=8)
plt.legend(['Actual', 'Anomaly Detected'])
plt.xlabel('Time Period')
plt.xticks([0, 20, 40, 60, 80, 99],[ts_dataframe.index[0],ts_dataframe.index[20], ts_dataframe.index[40], ts_dataframe.index[60], ts_dataframe.index[80], ts_dataframe.index[99]] ,rotation=45)
plt.ylabel('Normalized Profit')
# # Sets the figure size temporarily but has to be set again the next plot
# plt.figure(figsize=(20,16))
# sns.lineplot(x=tdf.index, y="value_hrf", hue="sensor", data=tdf);
# plt.show()
# tdf = tdf[tdf['sensor'].isin(['bmp180', 'htu21d', 'pr103j2', 'tsys01'])]
# Sets the figure size temporarily but has to be set again the next plot
# plt.figure(figsize=(20,16))
# sns.lineplot(x=tdf.index, y="value_hrf", hue="sensor", data=tdf);
# plt.show()
minidf.head(10)
len(minidf)
len(tdf)
non_tdf = minidf[minidf.parameter != 'temperature'].copy()
len(non_tdf)
for p in non_tdf.parameter.unique(): print(p)
# non_tdf.head(1000)
# i dont think we necessariry care too much about acceleration data (although its good to have), but
# maybe lets focus on
hdf = minidf[minidf.parameter == 'humidity'].copy()
hdf.head(10)
# three sep datavalues !
# Sets the figure size temporarily but has to be set again the next plot
plt.figure(figsize=(20,16))
sns.lineplot(x=hdf.index, y="value_hrf", hue="sensor", data=hdf);
plt.show()
hdf.dtypes
hdf = hdf[hdf['sensor'].isin(['hih4030'])]
# Sets the figure size temporarily but has to be set again the next plot
plt.figure(figsize=(20,16))
sns.lineplot(x=hdf.index, y="value_hrf", hue="sensor", data=hdf);
plt.show()
hdf.value_hrf = hdf.value_hrf.astype(np.float16)
hdf = hdf.loc['2019-02-01 03:00':'2019-02-01 04:00'].copy()
# Sets the figure size temporarily but has to be set again the next plot
plt.figure(figsize=(20,16))
sns.lineplot(x=hdf.index, y="value_hrf", hue="sensor", data=hdf);
plt.show()
hdf = minidf[minidf.parameter == 'humidity'].copy()
hdf.value_hrf = hdf.value_hrf.astype(np.float16)
hdf = hdf[hdf['sensor'].isin(['hih4030', 'htu21d'])]
hdf = hdf.loc['2019-02-01':'2019-02-12'].copy()
# Sets the figure size temporarily but has to be set again the next plot
plt.figure(figsize=(20,16))
sns.lineplot(x=hdf.index, y="value_hrf", hue="sensor", data=hdf);
plt.show()
minidf = df.loc['2019-01-01':'2019-06-01'].copy()
hdf = minidf[minidf.parameter == 'humidity'].copy()
hdf.value_hrf = hdf.value_hrf.astype(np.float16)
hdf = hdf[hdf['sensor'].isin(['htu21d'])]
# hdf = hdf.loc['2019-02-01':'2020-02-01'].copy()
# Sets the figure size temporarily but has to be set again the next plot
plt.figure(figsize=(20,16))
sns.lineplot(x=hdf.index, y="value_hrf", hue="sensor", data=hdf);
plt.show()
minidf = df.loc['2019-01-01':'2019-06-01'].copy()
hdf = minidf[minidf.parameter == 'humidity'].copy()
hdf.value_hrf = hdf.value_hrf.astype(np.float16)
hdf = hdf[hdf['sensor'].isin(['hih4030'])]
# hdf = hdf.loc['2019-02-01':'2020-02-01'].copy()
# Sets the figure size temporarily but has to be set again the next plot
plt.figure(figsize=(20,16))
sns.lineplot(x=hdf.index, y="value_hrf", hue="sensor", data=hdf);
plt.show()
# make large numbers more readable by adding commas
# pd.options.display.float_format = '{:,}'.format
# Example
# 123456789.12345 -> 123,456,789.12345